# -*- coding: utf-8 -*-
import json
import random
import scrapy


cate_info = {
    '教育':'363',
    '历史':'383',
    '体育':'382',
    '国际':'380',
    '军事':'377',
    '健康':'376',
    '电影':'375',
    '房产':'374',
    '明星':'373',
    '旅游':'372',
    '娱乐':'371',
    '汽车':'370',
    '美酒':'367',
    '家居':'366',
    '母婴':'359',
    '公益':'358',
    '财经':'158',
    '宠物':'159',
    '佛教':'161',
    '星座':'169',
    '科技':'163',
    '时尚':'166'
}
cate_fh = {
    '国际':'https://api.iclient.ifeng.com/nlist?id=GJPD&action=default&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551253849&sn=d55d8d530ffa178e3f23b7c506f4b78f',
    '明星':'https://api.3g.ifeng.com/clientChannelNewsSearch?k=%E6%98%8E%E6%98%9F&action=default&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551682396&sn=40f07b35b0952f062db3c8f389ad7772',
    '宠物':'https://api.iclient.ifeng.com/nlist?id=MENGCHONG&action=default&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551254052&sn=7293b232f9f9a3b2877b82d1990d96ea',
    '科技':'https://api.iclient.ifeng.com/nlist?id=KJ123&action=default&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551254479&sn=7f6d9d10764562eef82cd91fc1b58af3',
    '历史':'https://api.iclient.ifeng.com/nlist?id=LS153&action=down&pullNum=1&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551682024&sn=1252bb4845bba393be93d492e4095b74',
    '健康':'https://api.iclient.ifeng.com/nlist?id=JK36&action=default&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551681899&sn=61ae9438a1b88af5f6c5d29ecbd64c12',
    '电影':'https://api.iclient.ifeng.com/nlist?id=DYPD&action=default&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551253793&sn=6e2e612ad6d0539bbb53b79dd64a3483',
    '星座':'https://api.iclient.ifeng.com/nlist?id=XZ09&action=default&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&district=%E6%BB%A8%E6%B1%9F%E5%8C%BA&gv=6.3.9&av=6.3.9&uid=865873035293665&deviceid=865873035293665&proid=ifengnews&os=android_26&df=androidphone&vt=5&screen=1080x1920&publishid=6001&nw=wifi&loginid=&st=1551680695&sn=bcb6d86d49b40bf7f3eade0bb2a3af4c',
    '公益':'https://api.iclient.ifeng.com/nlist?id=GYPD%2CFOCUSGYPD&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551252121&sn=a2c40b4d88ed280c9cd7932c06f1bbe9',
    '财经':'https://api.iclient.ifeng.com/nlist?id=CJ33,FOCUSCJ33,HNCJ33&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253126&sn=279aa1e3dd6a109ea3a491c333f8d38b',
    '体育':'https://api.iclient.ifeng.com/nlist?id=TY43,FOCUSTY43,TYTOPIC&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253288&sn=ff0875a521885530e87be4951eacc917',
    '军事':'https://api.iclient.ifeng.com/nlist?id=JS83,FOCUSJS83&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253341&sn=ac662fab247fc77d377ece28e755e521',
    '佛教':'https://api.iclient.ifeng.com/nlist?id=FJ31,FOCUSFJ31,SECNAVFJ31&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253716&sn=ec4696bea2759cd7a8d51c935400dd92',
    '时尚':'https://api.iclient.ifeng.com/nlist?id=SS78,FOCUSSS78,SECNAVSS78&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253886&sn=f30bd62597189dc2c7c30a73ae98b60b',
    '旅游':'https://api.iclient.ifeng.com/nlist?id=LY67,FOCUSLY67,SECNAVLY67&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253907&sn=b89ccb789f3ed730070a0776d013b3a5',
    '家居':'https://api.iclient.ifeng.com/nlist?id=JJPD,FOCUSJJPD&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551254118&sn=2e4a359dee00e5669b21ddab14a9090c',
    '娱乐':'https://api.iclient.ifeng.com/nlist?id=YL53,FOCUSYL53,SECNAVYL53&action=down&pullNum=2&province=%E6%B5%99%E6%B1%9F%E7%9C%81&city=%E6%9D%AD%E5%B7%9E%E5%B8%82&sptype=1&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551255143&sn=0e64fa59f5b4570c945086da75c575fa&dlt=30.218812&dln=120.218617&dcy=%E6%9D%AD%E5%B7%9E%E5%B8%82&dpr=%E6%B5%99%E6%B1%9F%E7%9C%81',
    '汽车':'https://api.iclient.ifeng.com/ClientNewsRegion?id=QC45,FOCUSQC45,SECNAVQC45&action=default&choicename=%E5%85%A8%E5%9B%BD&choicetype=null&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253419&sn=ce0c0e328ae9bd6f925b562ce9a3d94d',
    '房产':'https://api.iclient.ifeng.com/ClientNewsRegion?id=FC81,FOCUSFC81,SECNAVFC81&action=default&choicename=%E6%9D%AD%E5%B7%9E&choicetype=city&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551253532&sn=7fa9c6605af294807a3d939e86a32a4f',
    '母婴':'https://api.iclient.ifeng.com/nlist?id=PARENT&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551254668&sn=20b94a938f3cad4889f78e0f505a0ee0',
    '美酒':'https://api.iclient.ifeng.com/nlist?id=JYPD,FOCUSJYPD&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551254169&sn=fb91e68c74b99805e38bf5794961d967',
    '教育':'https://api.3g.ifeng.com/clientChannelNewsSearch?k=%E6%95%99%E8%82%B2&gv=6.3.9&uid=865873035293665&proid=ifengnews&os=android_26&vt=5&publishid=6001&st=1551254834&sn=c7d18ce3341e5ae916e589931b78436d&dlt=30.218812&dln=120.218617&dcy=%E6%9D%AD%E5%B7%9E%E5%B8%82&dpr=%E6%B5%99%E6%B1%9F%E7%9C%81',
}

class FengHuangSpider(scrapy.Spider):
    name = 'fhxw'

    def start_requests(self):
        for cate,key in cate_fh.items():
            yield scrapy.Request(key, callback=self.parse, meta={"cate":cate})

    def parse(self, response):
        cate = response.meta["cate"]
        res = json.loads(response.text)
        for ret in res:
            if ret.get('type') == 'list':
                res_list = ret.get('item')
                for info in res_list:
                    if info.get('type')=='doc': #只要纯文
                        item = {}
                        item['title'] = info.get('title')
                        item['publish_time'] = info.get('updateTime').replace('/','-')
                        item['author'] = info.get('source')
                        item['comment_count'] = int(info.get('comments')) if info.get('comments') else 0
                        if item['comment_count'] != 0:
                            item['browse_count']=item['comment_count']*random.randint(100,200)
                        else:
                            item['browse_count']=random.randint(100,3000)
                        item['site_classify_id'] = cate_info.get(cate)
                        item['site_id'] = 9 #9表示凤凰网
                        # item['url'] = info.get('link').get('weburl') #手机url
                        item['url'] = 'https://feng.ifeng.com/c/'+info.get('id').split('_')[-1]
                        # print(item)
                        yield item

